#!/usr/bin/env python
# -*- coding:UTF-8 -*-

'''
Created on 2018年7月8日

@author: Administrator
'''

import requests
from bs4 import BeautifulSoup
import pandas as pd
import json
import time
from _ast import Index


# 网址：
# https://cd.lianjia.com/ershoufang/rs/

def main():
    excel_data_frame = []
    for pageIdx in range (1,11):
        url = "https://cd.lianjia.com/ershoufang/pg%s/" %pageIdx
        responseResult = requests.get(url).text
        soup = BeautifulSoup(responseResult,'html.parser')
        houseInfo = soup.findAll('div', {'class':'houseInfo'})
    #     print houseInfo
        houseName = [i.text.split('|')[0].strip() for i in houseInfo]
        houseType = [i.text.split('|')[1].strip() for i in houseInfo]
        houseSize = [i.text.split('|')[2].strip() for i in houseInfo]
        houseOrientations = [i.text.split('|')[3].strip() for i in houseInfo]
        houseDecorate = [i.text.split('|')[4].strip() for i in houseInfo] 
        hasLift = []
        for i in houseInfo:
            if len(i.text.split('|'))==6:
                hasLift.append(i.text.split('|')[5].strip());
            else:
                hasLift.append(None)
        
        positionInfo = soup.findAll('div', {'class':'positionInfo'})
#         floor = [i.text[:i.text.index(")")+1].strip() for i in positionInfo]
        floor = [i.text.strip() for i in positionInfo]
        position = [i.text.split('-')[1].strip() for i in positionInfo]
        
        totalPriceInfo = soup.findAll('div', {'class':'totalPrice'})
        totalPrice = [i.text.strip() for i in totalPriceInfo]
        
        unitPriceInfo = soup.findAll('div', {'class':'unitPrice'})
        unitPrice = [i.text.strip() for i in unitPriceInfo]
    
        # 生成表格
    #     excel_data_frame = pd.DataFrame(dict(houseName=houseName,houseType=houseType,
    #                       houseSize=houseSize,houseOrientations=houseOrientations,
    #                       houseDecorate=houseDecorate,
    #                       #hasLift=hasLift,
    #                       floor=floor,position=position))
        excel_data_frame_item = pd.DataFrame({'小区名称':houseName,'户型':houseType,
                          '总价':totalPrice,'单价':unitPrice,
                          '面积':houseSize,'朝向':houseOrientations,
                          '装修':houseDecorate,
                          '楼层':floor,'位置坐标':position,'是否有电梯':hasLift},
                          columns=['小区名称','总价','单价','面积','户型','朝向','装修','楼层','位置坐标','是否有电梯'])
        excel_data_frame.append(excel_data_frame_item)
    houseInfoExcel = pd.concat(excel_data_frame,ignore_index=True) # 将多个表格拼接到一起
    print houseInfoExcel
    houseInfoExcel.to_excel("houseInfo.xlsx",index=False) #生成为excel文件
    

main()